/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.openjawtech.xscraper.scraper.runtime;

import com.openjawtech.xscraper.schema.scrape_config.ScrapeGroupsType;
import com.openjawtech.xscraper.schema.scrape_config.ScrapeGroupType;
import com.openjawtech.xscraper.schema.scrape_config.ScrapesType;
import com.openjawtech.xscraper.schema.scrape_config.ScrapeType;
import com.openjawtech.xscraper.schema.scrape_config.TravelSimpleType;
import com.openjawtech.xscraper.schema.scrape_config.TravelType;
import com.openjawtech.xscraper.schema.scrape_config.ScrapeSimpleType;
import com.openjawtech.xscraper.schema.FlightsType;
import com.openjawtech.xscraper.schema.FlightType;
import com.openjawtech.xscraper.scraper.util.HtmlParserUtil;
import com.openjawtech.xscraper.scraper.util.StringUtil;
import com.openjawtech.xscraper.scraper.util.DateUtil;
import com.openjawtech.xscraper.scraper.runtime.xml.ScrapeXpath;
import com.openjawtech.xscraper.scraper.runtime.scripting.JSScripting;
import com.openjawtech.xscraper.scraper.exception.ScrapeException;
import java.io.File;
import java.io.InputStream;
import java.math.BigDecimal;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.w3c.dom.Document;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 *
 * @Created on 26-Apr-2009, 08:34:46
 * @author Wenbo.Sun
 */
public class HtmlScraper {

    private ScrapeGroupsType sgt;
    private Document doc;
    private ScrapeXpath xpath = new ScrapeXpath();
    private JSScripting jsspt = new JSScripting();
    private Log log = LogFactory.getLog(HtmlScraper.class);

    public HtmlScraper(ScrapeGroupsType sgt, File htmlFile) {
        this.sgt = sgt;
        doc = HtmlParserUtil.parse(htmlFile);
    }

    public HtmlScraper(ScrapeGroupsType sgt, Document doc) {
        this.sgt = sgt;
        this.doc = doc;
    }

    public HtmlScraper(ScrapeGroupsType sgt, InputStream is) {
        this.sgt = sgt;
        doc = HtmlParserUtil.parse(is);
    }

    public List<FlightsType> scrape() {
        log.debug("start scraping ...");
        List<FlightsType> list = new ArrayList<FlightsType>();
        for (ScrapeGroupType sg : sgt.getScrapeGroup()) {
            ScrapesType departureType = sg.getDeparture();
            String departure = singleStrScrape(departureType);
            if (departure == null) {
                continue;
            }
            ScrapesType arrivalType = sg.getArrival();
            String arrival = singleStrScrape(arrivalType);
            if (arrival == null) {
                continue;
            }
            FlightsType flightsType = FlightsType.Factory.newInstance();
            list.add(flightsType);
            flightsType.setDeparture(departure);
            flightsType.setArrival(arrival);

            log.debug("departure -- " + departure + " and arrival -- " + arrival);

            for (TravelType tt : sg.getTravel()) {
                try {
                    TravelSimpleType tst = tt.getType();
                    String dateFormat = tt.getDateformat();
                    ScrapesType dateType = tt.getTravelDate();
                    ScrapesType priceType = tt.getPrice();
                    String prefix = tt.getPrefix();
                    switch (tst) {
                        case SINGLE:
                            Calendar date = singleDateScrape(dateType, dateFormat);
                            BigDecimal price = singlePriceScrape(priceType);
                            FlightType flightType = flightsType.addNewFlight();
                            flightType.setPrice(price);
                            flightType.setTravelDate(date);
                            break;
                        case ARRAY:
                            List<Calendar> cals = arrayDateScrape(dateType, dateFormat);
                            List<BigDecimal> prices = arrayPriceScrape(priceType);
                            if (cals.size() != prices.size()) {
                                break;
                            }
                            for (int i = 0; i < cals.size(); i++) {
                                FlightType flight1Type = flightsType.addNewFlight();
                                flight1Type.setPrice(prices.get(i));
                                flight1Type.setTravelDate(cals.get(i));
                            }
                            break;
                        case MULTIPLE:
                            int size = xpath.evaluateNodes(prefix, doc).getLength();
                            for (int i = 1; i <= size; i++) {
                                Calendar mCal = multipleDateScrape(dateType, dateFormat, i, prefix);
                                BigDecimal mPrice = multiplePriceScrape(priceType, i, prefix);
                                FlightType flightType2 = flightsType.addNewFlight();
                                flightType2.setPrice(mPrice);
                                flightType2.setTravelDate(mCal);
                            }

                    }
                } catch (Throwable t) {
                    log.error("scrape travel info failed", t);
                }
            }
        }
        return list;
    }

    private String singleStrScrape(ScrapesType scrapesType) {
        String dest = null;
        for (ScrapeType st : scrapesType.getScrape()) {
            String path = st.getPath();
            ScrapeSimpleType sst = st.getType();
            switch (sst) {
                case HTML:
                    dest = singleStrHtmlScrape(path, st);
                    break;
                case JS:
                    dest = singleStrJsScrape(path, st);
                    break;
            }
            if (dest != null) {
                break;
            }
        }
        return dest;
    }

    private String singleStrHtmlScrape(String path, ScrapeType st) {
        String dest = xpath.evaluateStr(path, doc);
        String scanner = st.getScanner();
        dest = StringUtil.simpleFormat(dest);
        if (scanner == null) {
            return dest;
        }
        log.debug("scrape result -- " + dest + " scanner -- " + scanner);
        return StringUtil.getMatchResult(dest, scanner);
    }

    private String singleStrJsScrape(String path, ScrapeType st) {
        String js = xpath.evaluateStr(path, doc);
        String var = st.getVarname();
        String dest = (String) jsspt.getValue(var, js);
        return dest;
    }

    private Calendar singleDateScrape(ScrapesType scrapesType, String dateformat) {
        Date dest = null;
        String dateStr = singleStrScrape(scrapesType);
        try {
            dest = DateUtil.createDate(dateStr, dateformat);
        } catch (ParseException ex) {
            throw new ScrapeException("create date failed!", ex);
        }
        return DateUtil.createCalendar(dest);
    }

    private BigDecimal singlePriceScrape(ScrapesType scrapesType) {
        String priceStr = singleStrScrape(scrapesType);
        if (priceStr == null || "".equals(priceStr)) {
            throw new ScrapeException("retrieve price failed!");
        }
        double price = Double.parseDouble(priceStr);
        return BigDecimal.valueOf(price);
    }

    private List<Calendar> arrayDateScrape(ScrapesType scrapesType, String dateformat) {
        List<Calendar> cals = new ArrayList<Calendar>();
        for (ScrapeType st : scrapesType.getScrape()) {
            try {
                String js = xpath.evaluateStr(st.getPath(), doc);
                List<String> list = jsspt.getArrayValue(st.getVarname(), js);
                for (String str : list) {
                    Calendar date = DateUtil.createCalendar(DateUtil.createDate(str, dateformat));
                    cals.add(date);
                }
            } catch (Throwable t) {
                throw new ScrapeException(t);
            }
            if (cals.size() != 0) {
                break;
            }
        }
        return cals;
    }

    private List<BigDecimal> arrayPriceScrape(ScrapesType scrapesType) {
        List<BigDecimal> prices = new ArrayList<BigDecimal>();
        for (ScrapeType st : scrapesType.getScrape()) {
            try {
                String js = xpath.evaluateStr(st.getPath(), doc);
                List<String> list = jsspt.getArrayValue(st.getVarname(), js);
                for (String str : list) {
                    BigDecimal price = BigDecimal.valueOf(Double.parseDouble(str));
                    prices.add(price);
                }
            } catch (Throwable t) {
                throw new ScrapeException(t);
            }
            if (prices.size() != 0) {
                break;
            }
        }
        return prices;
    }

    private Calendar multipleDateScrape(ScrapesType scrapesType, String dateformat, int index, String prefix) {
        String dest = "";
        for (ScrapeType st : scrapesType.getScrape()) {
            String scanner = st.getScanner();
            String path = prefix + "[" + index + "]/" + st.getPath();
            dest = singleStrHtmlScrape(path, st);
            if (scanner != null || "".equals(scanner)) {
                dest = StringUtil.getMatchResult(StringUtil.simpleFormat(dest), scanner);

            }
            if (dest != null && !"".equals(dest)) {
                break;
            }
        }
        Date date = null;
        if (dest != null && !"".equals(dest)) {
            try {
                date = DateUtil.createDate(dest, dateformat);
            } catch (ParseException ex) {
                throw new ScrapeException("create date failed!", ex);
            }
            return DateUtil.createCalendar(date);
        }
        return null;
    }

    private BigDecimal multiplePriceScrape(ScrapesType scrapesType, int index, String prefix) {
        String dest = "";
        for (ScrapeType st : scrapesType.getScrape()) {
            String scanner = st.getScanner();
            String path = prefix + "[" + index + "]/" + st.getPath();
            dest = singleStrHtmlScrape(path, st);
            if (scanner != null && !"".equals(scanner)) {
                dest = StringUtil.getMatchResult(StringUtil.simpleFormat(dest), scanner);
            }
            if (dest != null && !"".equals(dest)) {
                break;
            }
        }
        if (dest != null && !"".equals(dest)) {
            double price = Double.parseDouble(dest);
            return BigDecimal.valueOf(price);
        }
        return BigDecimal.valueOf(-1);
    }
}
